{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation.evaluate import evaluate\n",
    "from data_utils.basic_data_round2 import load_train_val_dataset,load_basic_dataset\n",
    "from results_process.regulizer import remove_nine,remove_short_entity\n",
    "from results_process.utils import load_model_rs\n",
    "from results_process.bert_entity_model import reduce_rs_by_id\n",
    "from functools import reduce\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "from glob import glob\n",
    "from os.path import join\n",
    "from tqdm import tqdm\n",
    "MODEL_ROOT = '/home/njuciairs/wangshuai/competitions/finacial_models_round2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/home/njuciairs/wangshuai/competitions/finacial_models_round2\u001b[00m\r\n",
      "├── \u001b[01;34mBertSentiEntity_cross\u001b[00m\r\n",
      "├── \u001b[01;34mbinary_class_cross_roberta\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573410905.AIRS.24109.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561033.AIRS.23888.12\r\n",
      "│   │   │       ├── events.out.tfevents.1573561463.AIRS.23888.21\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573410910.AIRS.24109.1\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573414365.AIRS.24109.4\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573414365.AIRS.24109.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573412612.AIRS.24109.2\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573446913.AIRS.7323.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561084.AIRS.23888.13\r\n",
      "│   │   │       ├── events.out.tfevents.1573561865.AIRS.23888.22\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573446917.AIRS.7323.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573448534.AIRS.7323.2\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573467021.AIRS.26865.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561134.AIRS.23888.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573562266.AIRS.23888.23\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573467025.AIRS.26865.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573468955.AIRS.26865.2\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573478138.AIRS.26865.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573561183.AIRS.23888.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573562668.AIRS.23888.24\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573478140.AIRS.26865.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573479810.AIRS.26865.5\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573490017.AIRS.26865.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573561234.AIRS.23888.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573563069.AIRS.23888.25\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573490019.AIRS.26865.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573491923.AIRS.26865.8\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573526921.AIRS.23888.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561280.AIRS.23888.17\r\n",
      "│   │   │       ├── events.out.tfevents.1573563470.AIRS.23888.26\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573526925.AIRS.23888.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573528553.AIRS.23888.2\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573537414.AIRS.23888.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573561325.AIRS.23888.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573563871.AIRS.23888.27\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573537417.AIRS.23888.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573538960.AIRS.23888.5\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573543886.AIRS.23888.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573561371.AIRS.23888.19\r\n",
      "│   │   │       ├── events.out.tfevents.1573564273.AIRS.23888.28\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573543888.AIRS.23888.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573545363.AIRS.23888.8\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573553232.AIRS.23888.9\r\n",
      "│   │           ├── events.out.tfevents.1573561415.AIRS.23888.20\r\n",
      "│   │           ├── events.out.tfevents.1573564674.AIRS.23888.29\r\n",
      "│   │           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573553234.AIRS.23888.10\r\n",
      "│   │           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573554689.AIRS.23888.11\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.979913_epoch7_time20191111054906\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.972230_epoch7_time20191111154518\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.964897_epoch5_time20191111203714\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.976054_epoch3_time20191111224656\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.976327_epoch3_time20191112020819\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.980947_epoch5_time20191112130505\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.977693_epoch1_time20191112140920\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.970953_epoch5_time20191112173350\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.974504_epoch2_time20191112185543\r\n",
      "├── \u001b[01;34mmulti_class_cross1\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573026942.AIRS.13972.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573196075.AIRS.15210.27\r\n",
      "│   │   │       ├── events.out.tfevents.1573398630.AIRS.15463.0\r\n",
      "│   │   │       └── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573026950.AIRS.13972.1\r\n",
      "│   │   ├── \u001b[01;34m10\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573180302.AIRS.15210.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573399967.AIRS.22163.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573412781.AIRS.28509.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573180305.AIRS.15210.25\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573181396.AIRS.15210.26\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573027124.AIRS.15210.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573398766.AIRS.16498.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573399702.AIRS.22163.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573400088.AIRS.22163.9\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573027132.AIRS.15210.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573028203.AIRS.15210.2\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573043806.AIRS.15210.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573398804.AIRS.16498.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573399735.AIRS.22163.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573400605.AIRS.22163.10\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573043812.AIRS.15210.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573044886.AIRS.15210.5\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573057365.AIRS.15210.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573398863.AIRS.16498.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573399770.AIRS.22163.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573401041.AIRS.22163.11\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573057369.AIRS.15210.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573058449.AIRS.15210.8\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573086938.AIRS.15210.9\r\n",
      "│   │   │       ├── events.out.tfevents.1573398960.AIRS.16498.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573399802.AIRS.22163.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573402451.AIRS.22163.12\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573086942.AIRS.15210.10\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573088023.AIRS.15210.11\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573100391.AIRS.15210.12\r\n",
      "│   │   │       ├── events.out.tfevents.1573398996.AIRS.16498.4\r\n",
      "│   │   │       ├── events.out.tfevents.1573399838.AIRS.22163.4\r\n",
      "│   │   │       ├── events.out.tfevents.1573402819.AIRS.22163.13\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573100395.AIRS.15210.13\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573101479.AIRS.15210.14\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573118523.AIRS.15210.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573399137.AIRS.16498.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573399869.AIRS.22163.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573403212.AIRS.22163.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573411602.AIRS.28086.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573411678.AIRS.28509.0\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573118530.AIRS.15210.16\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573119596.AIRS.15210.17\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573135028.AIRS.15210.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573399188.AIRS.16498.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573399902.AIRS.22163.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573412046.AIRS.28509.1\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573135031.AIRS.15210.19\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573136118.AIRS.15210.20\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573157183.AIRS.15210.21\r\n",
      "│   │           ├── events.out.tfevents.1573399226.AIRS.16498.7\r\n",
      "│   │           ├── events.out.tfevents.1573399932.AIRS.22163.7\r\n",
      "│   │           ├── events.out.tfevents.1573412413.AIRS.28509.2\r\n",
      "│   │           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573157188.AIRS.15210.22\r\n",
      "│   │           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573158267.AIRS.15210.23\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mdata\u001b[00m\r\n",
      "│   ├── \u001b[01;34mversion10\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.918750_epoch4_time20191108114352\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.908206_epoch5_time20191106172749\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.921625_epoch2_time20191106211303\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.902092_epoch7_time20191107022822\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.928776_epoch2_time20191107091135\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.920174_epoch6_time20191107140826\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.923510_epoch5_time20191107185117\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.933245_epoch9_time20191108003930\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.916217_epoch3_time20191108050017\r\n",
      "├── \u001b[01;34mmulti_class_cross_roberta\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573408955.AIRS.12503.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573534379.AIRS.27396.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573534668.AIRS.27396.27\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573408962.AIRS.12503.1\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573411125.AIRS.12503.4\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573411125.AIRS.12503.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573410044.AIRS.12503.2\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573424990.AIRS.8891.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573534413.AIRS.27396.19\r\n",
      "│   │   │       ├── events.out.tfevents.1573535035.AIRS.27396.28\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573424997.AIRS.8891.1\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573427150.AIRS.8891.4\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573427150.AIRS.8891.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573426074.AIRS.8891.2\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573438279.AIRS.8891.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573534447.AIRS.27396.20\r\n",
      "│   │   │       ├── events.out.tfevents.1573535400.AIRS.27396.29\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573438282.AIRS.8891.6\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573440444.AIRS.8891.9\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573440444.AIRS.8891.8\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573439362.AIRS.8891.7\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573461523.AIRS.27396.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573534481.AIRS.27396.21\r\n",
      "│   │   │       ├── events.out.tfevents.1573535766.AIRS.27396.30\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461530.AIRS.27396.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573462629.AIRS.27396.2\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573475114.AIRS.27396.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573534515.AIRS.27396.22\r\n",
      "│   │   │       ├── events.out.tfevents.1573536131.AIRS.27396.31\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573475118.AIRS.27396.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573476202.AIRS.27396.5\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573487672.AIRS.27396.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573534548.AIRS.27396.23\r\n",
      "│   │   │       ├── events.out.tfevents.1573536497.AIRS.27396.32\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573487675.AIRS.27396.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573488770.AIRS.27396.8\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573500215.AIRS.27396.9\r\n",
      "│   │   │       ├── events.out.tfevents.1573534579.AIRS.27396.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573536862.AIRS.27396.33\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573500218.AIRS.27396.10\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573501323.AIRS.27396.11\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573511761.AIRS.27396.12\r\n",
      "│   │   │       ├── events.out.tfevents.1573534608.AIRS.27396.25\r\n",
      "│   │   │       ├── events.out.tfevents.1573537228.AIRS.27396.34\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573511764.AIRS.27396.13\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573512864.AIRS.27396.14\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573519854.AIRS.27396.15\r\n",
      "│   │           ├── events.out.tfevents.1573534638.AIRS.27396.26\r\n",
      "│   │           ├── events.out.tfevents.1573537594.AIRS.27396.35\r\n",
      "│   │           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573519857.AIRS.27396.16\r\n",
      "│   │           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573520937.AIRS.27396.17\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.922166_epoch3_time20191111030235\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.926959_epoch6_time20191111082927\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.915481_epoch5_time20191111115139\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.936144_epoch7_time20191111184620\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.917137_epoch6_time20191111221356\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.921900_epoch6_time20191112014343\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.932367_epoch5_time20191112045514\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.907922_epoch2_time20191112071221\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.919583_epoch8_time20191112111410\r\n",
      "├── \u001b[01;34msenti_entity_goodremove\u001b[00m\r\n",
      "├── \u001b[01;34msenti_entity_goodremove_full\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573461283.AIRS.25939.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573470493.AIRS.25939.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573479455.AIRS.1625.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573479466.AIRS.1625.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573487448.AIRS.1625.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573495423.AIRS.1625.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573502486.AIRS.1625.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573511739.AIRS.1625.32\r\n",
      "│   │   │       ├── events.out.tfevents.1573511923.AIRS.1625.34\r\n",
      "│   │   │       ├── events.out.tfevents.1573512024.AIRS.1625.39\r\n",
      "│   │   │       ├── events.out.tfevents.1573544153.AIRS.27176.0\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461292.AIRS.25939.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461292.AIRS.25939.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461944.AIRS.25939.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461944.AIRS.25939.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573461944.AIRS.25939.5\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573479653.AIRS.1625.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573487454.AIRS.1625.9\r\n",
      "│   │   │       ├── events.out.tfevents.1573511948.AIRS.1625.35\r\n",
      "│   │   │       ├── events.out.tfevents.1573512194.AIRS.1625.40\r\n",
      "│   │   │       ├── events.out.tfevents.1573544208.AIRS.27176.1\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573479660.AIRS.1625.4\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573479660.AIRS.1625.3\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573480313.AIRS.1625.6\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573480313.AIRS.1625.5\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573480313.AIRS.1625.7\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573487632.AIRS.1625.10\r\n",
      "│   │   │       ├── events.out.tfevents.1573495428.AIRS.1625.17\r\n",
      "│   │   │       ├── events.out.tfevents.1573511973.AIRS.1625.36\r\n",
      "│   │   │       ├── events.out.tfevents.1573512364.AIRS.1625.41\r\n",
      "│   │   │       ├── events.out.tfevents.1573544261.AIRS.27176.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573487639.AIRS.1625.12\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573487639.AIRS.1625.11\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573488292.AIRS.1625.14\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573488292.AIRS.1625.13\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573488292.AIRS.1625.15\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573495862.AIRS.1625.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573502491.AIRS.1625.25\r\n",
      "│   │   │       ├── events.out.tfevents.1573511990.AIRS.1625.37\r\n",
      "│   │   │       ├── events.out.tfevents.1573512534.AIRS.1625.42\r\n",
      "│   │   │       ├── events.out.tfevents.1573544316.AIRS.27176.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573495996.AIRS.1625.20\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573495996.AIRS.1625.19\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573496649.AIRS.1625.22\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573496649.AIRS.1625.21\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573496649.AIRS.1625.23\r\n",
      "│   │   └── \u001b[01;34m5\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573502668.AIRS.1625.26\r\n",
      "│   │           ├── events.out.tfevents.1573511745.AIRS.1625.33\r\n",
      "│   │           ├── events.out.tfevents.1573512007.AIRS.1625.38\r\n",
      "│   │           ├── events.out.tfevents.1573512704.AIRS.1625.43\r\n",
      "│   │           ├── events.out.tfevents.1573544399.AIRS.27176.4\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573502674.AIRS.1625.28\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573502674.AIRS.1625.27\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573503328.AIRS.1625.30\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573503328.AIRS.1625.29\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573503328.AIRS.1625.31\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.963447_epoch8_time20191111180125\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.962730_epoch8_time20191111230740\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.962609_epoch8_time20191112012036\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.961126_epoch6_time20191112031816\r\n",
      "│   └── \u001b[01;34mversion5\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.961709_epoch8_time20191112053105\r\n",
      "├── \u001b[01;34msenti_entity_goodremove_on_round1data\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573213491.AIRS.30894.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573219507.AIRS.30894.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573219518.AIRS.30894.7\r\n",
      "│   │   │       ├── events.out.tfevents.1573224890.AIRS.30894.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573233218.AIRS.30894.22\r\n",
      "│   │   │       ├── events.out.tfevents.1573239879.AIRS.30894.30\r\n",
      "│   │   │       ├── events.out.tfevents.1573247885.AIRS.30894.38\r\n",
      "│   │   │       ├── events.out.tfevents.1573255073.AIRS.30894.46\r\n",
      "│   │   │       ├── events.out.tfevents.1573259572.AIRS.30894.54\r\n",
      "│   │   │       ├── events.out.tfevents.1573266749.AIRS.30894.62\r\n",
      "│   │   │       ├── events.out.tfevents.1573272065.AIRS.30894.70\r\n",
      "│   │   │       ├── events.out.tfevents.1573272254.AIRS.30894.72\r\n",
      "│   │   │       ├── events.out.tfevents.1573398902.AIRS.17318.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573399771.AIRS.22565.0\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213505.AIRS.30894.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213505.AIRS.30894.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213778.AIRS.30894.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213778.AIRS.30894.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573213778.AIRS.30894.5\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573219700.AIRS.30894.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573224902.AIRS.30894.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573272277.AIRS.30894.73\r\n",
      "│   │   │       ├── events.out.tfevents.1573399010.AIRS.17318.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573399951.AIRS.22565.1\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219704.AIRS.30894.10\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219704.AIRS.30894.9\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219980.AIRS.30894.12\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219980.AIRS.30894.11\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573219980.AIRS.30894.13\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573225083.AIRS.30894.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573233222.AIRS.30894.23\r\n",
      "│   │   │       ├── events.out.tfevents.1573272299.AIRS.30894.74\r\n",
      "│   │   │       ├── events.out.tfevents.1573399088.AIRS.17318.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573400128.AIRS.22565.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225088.AIRS.30894.18\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225088.AIRS.30894.17\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225364.AIRS.30894.20\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225364.AIRS.30894.19\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573225364.AIRS.30894.21\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573233398.AIRS.30894.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573239884.AIRS.30894.31\r\n",
      "│   │   │       ├── events.out.tfevents.1573272322.AIRS.30894.75\r\n",
      "│   │   │       ├── events.out.tfevents.1573399140.AIRS.17318.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573400303.AIRS.22565.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233402.AIRS.30894.26\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233402.AIRS.30894.25\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233675.AIRS.30894.28\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233675.AIRS.30894.27\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573233675.AIRS.30894.29\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573240060.AIRS.30894.32\r\n",
      "│   │   │       ├── events.out.tfevents.1573247889.AIRS.30894.39\r\n",
      "│   │   │       ├── events.out.tfevents.1573272345.AIRS.30894.76\r\n",
      "│   │   │       ├── events.out.tfevents.1573399428.AIRS.17318.4\r\n",
      "│   │   │       ├── events.out.tfevents.1573400478.AIRS.22565.4\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240064.AIRS.30894.34\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240064.AIRS.30894.33\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240336.AIRS.30894.36\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240336.AIRS.30894.35\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573240336.AIRS.30894.37\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573248064.AIRS.30894.40\r\n",
      "│   │   │       ├── events.out.tfevents.1573255076.AIRS.30894.47\r\n",
      "│   │   │       ├── events.out.tfevents.1573272368.AIRS.30894.77\r\n",
      "│   │   │       ├── events.out.tfevents.1573399478.AIRS.17318.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573400664.AIRS.22565.5\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248068.AIRS.30894.42\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248068.AIRS.30894.41\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248341.AIRS.30894.44\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248341.AIRS.30894.43\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573248341.AIRS.30894.45\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573255252.AIRS.30894.48\r\n",
      "│   │   │       ├── events.out.tfevents.1573259575.AIRS.30894.55\r\n",
      "│   │   │       ├── events.out.tfevents.1573272383.AIRS.30894.78\r\n",
      "│   │   │       ├── events.out.tfevents.1573399507.AIRS.17318.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573401300.AIRS.22565.6\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255256.AIRS.30894.50\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255256.AIRS.30894.49\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255528.AIRS.30894.52\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255528.AIRS.30894.51\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573255528.AIRS.30894.53\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573259751.AIRS.30894.56\r\n",
      "│   │   │       ├── events.out.tfevents.1573266753.AIRS.30894.63\r\n",
      "│   │   │       ├── events.out.tfevents.1573272397.AIRS.30894.79\r\n",
      "│   │   │       ├── events.out.tfevents.1573399531.AIRS.17318.7\r\n",
      "│   │   │       ├── events.out.tfevents.1573401485.AIRS.22565.7\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573259756.AIRS.30894.58\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573259755.AIRS.30894.57\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573260028.AIRS.30894.60\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573260028.AIRS.30894.59\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573260028.AIRS.30894.61\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573266930.AIRS.30894.64\r\n",
      "│   │           ├── events.out.tfevents.1573272069.AIRS.30894.71\r\n",
      "│   │           ├── events.out.tfevents.1573272411.AIRS.30894.80\r\n",
      "│   │           ├── events.out.tfevents.1573399554.AIRS.17318.8\r\n",
      "│   │           ├── events.out.tfevents.1573401681.AIRS.22565.8\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573266936.AIRS.30894.66\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573266936.AIRS.30894.65\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573267208.AIRS.30894.68\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573267208.AIRS.30894.67\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573267208.AIRS.30894.69\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.935708_epoch5_time20191108200756\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.965252_epoch9_time20191108220927\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.951920_epoch9_time20191108233913\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.952887_epoch5_time20191109013920\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.949487_epoch6_time20191109033451\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.943984_epoch4_time20191109053914\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.953871_epoch6_time20191109074801\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.925111_epoch6_time20191109090301\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.955598_epoch9_time20191109111611\r\n",
      "├── \u001b[01;34msenti_entity_inplace_full\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   └── \u001b[01;34m1\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573711871.AIRS.6990.0\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573711877.AIRS.6990.2\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573711877.AIRS.6990.1\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573712531.AIRS.6990.4\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573712531.AIRS.6990.3\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573712531.AIRS.6990.5\r\n",
      "│   └── \u001b[01;34mversion1\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       └── score0.962004_epoch7_time20191114152737\r\n",
      "├── \u001b[01;34msenti_entity_inplace_full_v1\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573727231.AIRS.1640.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573734958.AIRS.14407.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573734967.AIRS.14407.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573735142.AIRS.16060.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573735162.AIRS.16060.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573801084.AIRS.8283.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573810723.AIRS.8283.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573818260.AIRS.8283.22\r\n",
      "│   │   │       ├── events.out.tfevents.1573828433.AIRS.8283.30\r\n",
      "│   │   │       ├── events.out.tfevents.1573828644.AIRS.8283.32\r\n",
      "│   │   │       ├── events.out.tfevents.1573884890.AIRS.21602.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573885011.AIRS.21741.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573885718.AIRS.21741.5\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727236.AIRS.1640.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727236.AIRS.1640.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727916.AIRS.1640.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727916.AIRS.1640.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573727916.AIRS.1640.5\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573791715.AIRS.8283.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573801229.AIRS.8283.7\r\n",
      "│   │   │       ├── events.out.tfevents.1573885099.AIRS.21741.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573886317.AIRS.21741.6\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573791729.AIRS.8283.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573791729.AIRS.8283.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573792397.AIRS.8283.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573792397.AIRS.8283.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573792397.AIRS.8283.5\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573801413.AIRS.8283.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573810731.AIRS.8283.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573885194.AIRS.21741.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573801421.AIRS.8283.10\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573801421.AIRS.8283.9\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573802091.AIRS.8283.12\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573802091.AIRS.8283.11\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573802091.AIRS.8283.13\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573810914.AIRS.8283.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573818293.AIRS.8283.23\r\n",
      "│   │   │       ├── events.out.tfevents.1573885314.AIRS.21741.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573810935.AIRS.8283.18\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573810935.AIRS.8283.17\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573811601.AIRS.8283.20\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573811601.AIRS.8283.19\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573811601.AIRS.8283.21\r\n",
      "│   │   └── \u001b[01;34m5\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573818492.AIRS.8283.24\r\n",
      "│   │           ├── events.out.tfevents.1573828443.AIRS.8283.31\r\n",
      "│   │           ├── events.out.tfevents.1573885555.AIRS.21741.4\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573818506.AIRS.8283.26\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573818506.AIRS.8283.25\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573819175.AIRS.8283.28\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573819175.AIRS.8283.27\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573819175.AIRS.8283.29\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.967381_epoch7_time20191114194541\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   ├── score0.951676_epoch1_time20191115123317\r\n",
      "│   │   ├── score0.956579_epoch2_time20191115124426\r\n",
      "│   │   ├── score0.960477_epoch3_time20191115125534\r\n",
      "│   │   ├── score0.963445_epoch4_time20191115130640\r\n",
      "│   │   ├── score0.964204_epoch5_time20191115131746\r\n",
      "│   │   ├── score0.965986_epoch7_time20191115133956\r\n",
      "│   │   ├── score0.968415_epoch8_time20191115135102\r\n",
      "│   │   └── score0.970270_epoch10_time20191115141311\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   ├── score0.943036_epoch1_time20191115151451\r\n",
      "│   │   ├── score0.958119_epoch2_time20191115152601\r\n",
      "│   │   ├── score0.958387_epoch3_time20191115153709\r\n",
      "│   │   ├── score0.961559_epoch4_time20191115154816\r\n",
      "│   │   ├── score0.961809_epoch5_time20191115155923\r\n",
      "│   │   ├── score0.963421_epoch7_time20191115162131\r\n",
      "│   │   ├── score0.963601_epoch9_time20191115164335\r\n",
      "│   │   └── score0.966205_epoch10_time20191115165438\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   ├── score0.957020_epoch1_time20191115175321\r\n",
      "│   │   ├── score0.959496_epoch2_time20191115180427\r\n",
      "│   │   ├── score0.967964_epoch3_time20191115181532\r\n",
      "│   │   └── score0.969034_epoch7_time20191115185937\r\n",
      "│   └── \u001b[01;34mversion5\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       ├── score0.951851_epoch1_time20191115195935\r\n",
      "│       ├── score0.959211_epoch2_time20191115201041\r\n",
      "│       ├── score0.959282_epoch3_time20191115202146\r\n",
      "│       ├── score0.964579_epoch4_time20191115203250\r\n",
      "│       ├── score0.964818_epoch5_time20191115204353\r\n",
      "│       ├── score0.966388_epoch8_time20191115211656\r\n",
      "│       └── score0.966862_epoch11_time20191115214954\r\n",
      "└── \u001b[01;34msenti_entity_inplace_hit_full_v1\u001b[00m\r\n",
      "    ├── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573791469.AIRS.7248.0\r\n",
      "    │   │       ├── events.out.tfevents.1573801168.AIRS.7248.6\r\n",
      "    │   │       ├── events.out.tfevents.1573801176.AIRS.7248.7\r\n",
      "    │   │       ├── events.out.tfevents.1573811018.AIRS.7248.14\r\n",
      "    │   │       ├── events.out.tfevents.1573820841.AIRS.7248.22\r\n",
      "    │   │       ├── events.out.tfevents.1573830661.AIRS.7248.30\r\n",
      "    │   │       ├── events.out.tfevents.1573840552.AIRS.7248.38\r\n",
      "    │   │       ├── events.out.tfevents.1573840724.AIRS.7248.40\r\n",
      "    │   │       ├── events.out.tfevents.1573885031.AIRS.22007.0\r\n",
      "    │   │       ├── events.out.tfevents.1573885435.AIRS.22007.5\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573791475.AIRS.7248.2\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573791475.AIRS.7248.1\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573792125.AIRS.7248.4\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573792125.AIRS.7248.3\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573792125.AIRS.7248.5\r\n",
      "    │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573801350.AIRS.7248.8\r\n",
      "    │   │       ├── events.out.tfevents.1573811021.AIRS.7248.15\r\n",
      "    │   │       ├── events.out.tfevents.1573885117.AIRS.22007.1\r\n",
      "    │   │       ├── events.out.tfevents.1573885740.AIRS.22007.6\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573801353.AIRS.7248.10\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573801353.AIRS.7248.9\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573802005.AIRS.7248.12\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573802005.AIRS.7248.11\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573802005.AIRS.7248.13\r\n",
      "    │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573811191.AIRS.7248.16\r\n",
      "    │   │       ├── events.out.tfevents.1573820844.AIRS.7248.23\r\n",
      "    │   │       ├── events.out.tfevents.1573885208.AIRS.22007.2\r\n",
      "    │   │       ├── events.out.tfevents.1573885913.AIRS.22007.7\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811194.AIRS.7248.18\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811194.AIRS.7248.17\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811842.AIRS.7248.20\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811842.AIRS.7248.19\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573811842.AIRS.7248.21\r\n",
      "    │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573821014.AIRS.7248.24\r\n",
      "    │   │       ├── events.out.tfevents.1573830664.AIRS.7248.31\r\n",
      "    │   │       ├── events.out.tfevents.1573885326.AIRS.22007.3\r\n",
      "    │   │       ├── events.out.tfevents.1573886207.AIRS.22007.8\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821017.AIRS.7248.26\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821017.AIRS.7248.25\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821665.AIRS.7248.28\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821665.AIRS.7248.27\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573821665.AIRS.7248.29\r\n",
      "    │   └── \u001b[01;34m5\u001b[00m\r\n",
      "    │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │           ├── events.out.tfevents.1573830834.AIRS.7248.32\r\n",
      "    │           ├── events.out.tfevents.1573840554.AIRS.7248.39\r\n",
      "    │           ├── events.out.tfevents.1573885376.AIRS.22007.4\r\n",
      "    │           ├── events.out.tfevents.1573886549.AIRS.22007.9\r\n",
      "    │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573830837.AIRS.7248.34\r\n",
      "    │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573830837.AIRS.7248.33\r\n",
      "    │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573831488.AIRS.7248.36\r\n",
      "    │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573831488.AIRS.7248.35\r\n",
      "    │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │               └── events.out.tfevents.1573831488.AIRS.7248.37\r\n",
      "    ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.929784_epoch1_time20191115122845\r\n",
      "    │   ├── score0.942125_epoch2_time20191115123935\r\n",
      "    │   ├── score0.945590_epoch3_time20191115125024\r\n",
      "    │   ├── score0.952973_epoch4_time20191115130111\r\n",
      "    │   ├── score0.955620_epoch6_time20191115132242\r\n",
      "    │   ├── score0.955754_epoch7_time20191115133327\r\n",
      "    │   ├── score0.957575_epoch8_time20191115134413\r\n",
      "    │   ├── score0.958603_epoch9_time20191115135459\r\n",
      "    │   ├── score0.960612_epoch10_time20191115140544\r\n",
      "    │   ├── score0.960965_epoch11_time20191115141628\r\n",
      "    │   └── score0.963365_epoch12_time20191115142713\r\n",
      "    ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.917796_epoch1_time20191115151325\r\n",
      "    │   ├── score0.938023_epoch2_time20191115152415\r\n",
      "    │   ├── score0.947443_epoch4_time20191115154550\r\n",
      "    │   ├── score0.949963_epoch5_time20191115155638\r\n",
      "    │   ├── score0.951326_epoch6_time20191115160724\r\n",
      "    │   ├── score0.955394_epoch7_time20191115161808\r\n",
      "    │   ├── score0.956432_epoch10_time20191115165014\r\n",
      "    │   ├── score0.956660_epoch12_time20191115171136\r\n",
      "    │   ├── score0.957511_epoch13_time20191115172216\r\n",
      "    │   └── score0.957539_epoch15_time20191115174335\r\n",
      "    ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.916125_epoch1_time20191115175722\r\n",
      "    │   ├── score0.934901_epoch2_time20191115180808\r\n",
      "    │   ├── score0.939683_epoch3_time20191115181853\r\n",
      "    │   ├── score0.945166_epoch4_time20191115182936\r\n",
      "    │   ├── score0.946322_epoch5_time20191115184019\r\n",
      "    │   ├── score0.947342_epoch6_time20191115185102\r\n",
      "    │   ├── score0.947905_epoch7_time20191115190144\r\n",
      "    │   ├── score0.950549_epoch8_time20191115191224\r\n",
      "    │   ├── score0.953166_epoch9_time20191115192305\r\n",
      "    │   └── score0.954355_epoch12_time20191115195505\r\n",
      "    ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.928969_epoch1_time20191115204105\r\n",
      "    │   ├── score0.945035_epoch2_time20191115205152\r\n",
      "    │   ├── score0.947728_epoch3_time20191115210236\r\n",
      "    │   ├── score0.952182_epoch4_time20191115211320\r\n",
      "    │   ├── score0.952547_epoch5_time20191115212402\r\n",
      "    │   ├── score0.957946_epoch6_time20191115213444\r\n",
      "    │   ├── score0.958981_epoch10_time20191115221727\r\n",
      "    │   └── score0.961139_epoch11_time20191115222806\r\n",
      "    └── \u001b[01;34mversion5\u001b[00m\r\n",
      "        ├── \u001b[01;34mdata\u001b[00m\r\n",
      "        │   └── raw_rs.csv\r\n",
      "        ├── score0.911120_epoch1_time20191115232448\r\n",
      "        ├── score0.930853_epoch2_time20191115233538\r\n",
      "        ├── score0.932747_epoch3_time20191115234626\r\n",
      "        ├── score0.946900_epoch5_time20191116000804\r\n",
      "        ├── score0.948592_epoch7_time20191116002936\r\n",
      "        ├── score0.952430_epoch10_time20191116010156\r\n",
      "        ├── score0.955241_epoch11_time20191116011242\r\n",
      "        ├── score0.955925_epoch12_time20191116012330\r\n",
      "        └── score0.958125_epoch15_time20191116015549\r\n",
      "\r\n",
      "418 directories, 589 files\r\n"
     ]
    }
   ],
   "source": [
    "!tree $MODEL_ROOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic dataset len: 12999\n"
     ]
    }
   ],
   "source": [
    "train = load_basic_dataset('train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "negtive_samples = train[train['negative']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic dataset len: 9997\n"
     ]
    }
   ],
   "source": [
    "testdf = load_basic_dataset('test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#regulation1:是否共同出现的entity具备相同的标签\n",
    "\n",
    "#统计entity的标签分布情况\n",
    "def get_entity_labels(train_df):\n",
    "    \"\"\"\n",
    "    return dict[id,entity]->label\n",
    "    \"\"\"\n",
    "    rs_dict = {}\n",
    "    df = train_df[['id','entity','key_entity']]\n",
    "    for id,estr,kestr in df.values:\n",
    "        if not isinstance(estr,str):\n",
    "            continue\n",
    "        kestr = str(kestr).split(';')\n",
    "        for e in estr.split(';'):\n",
    "            if e in kestr:\n",
    "                rs_dict[(id,e)] = 1\n",
    "            else:\n",
    "                rs_dict[(id,e)] = 0\n",
    "    return rs_dict\n",
    "entity_label_dict = get_entity_labels(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_neighbour(entity1,entity2,text,title):\n",
    "    text = str(text)+str(title)\n",
    "    text = re.sub('[^A-Za-z0-9\\u4e00-\\u9fa5]', '', text)\n",
    "    estr1 = re.sub('[^A-Za-z0-9\\u4e00-\\u9fa5]', '', entity1+entity2)\n",
    "    estr2 = re.sub('[^A-Za-z0-9\\u4e00-\\u9fa5]', '', entity2+entity1)\n",
    "    if estr1 in text or estr2 in text:\n",
    "        return True\n",
    "    return False\n",
    "def exchange(e1,e2):\n",
    "    if e1 < e2:\n",
    "        return (e1,e2)\n",
    "    else:\n",
    "        return (e2,e1)\n",
    "\n",
    "#获取所有相邻的实体\n",
    "def get_all_neibours(train_df):\n",
    "    rs_list = []\n",
    "    df = train_df[['id','entity','text','title']]\n",
    "    for id,estr,text,title in df.values:\n",
    "        if not isinstance(estr,str):\n",
    "            continue\n",
    "        for e in estr.split(';'):\n",
    "            for e1 in estr.split(';'):\n",
    "                if  is_neighbour(e,e1,text,title):\n",
    "                    e,e1 = exchange(e,e1)\n",
    "                    rs_list.append((id,e,e1))\n",
    "    return list(set(rs_list))\n",
    "neibour_list = get_all_neibours(train[train['negative']==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#统计neigbours标签相同的数量和不同的数量\n",
    "same_neighbours = [(id,e1,e2) for id,e1,e2 in neibour_list if (entity_label_dict[(id,e1)] == entity_label_dict[(id,e2)])]\n",
    "diff_neighbours = [(id,e1,e2) for id,e1,e2 in neibour_list if (entity_label_dict[(id,e1)] != entity_label_dict[(id,e2)])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdf = testdf.sort_values(['id']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_rs['entity'] = testdf['entity']\n",
    "test_entity_label_dict = get_entity_labels(test_rs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_neighbours = get_all_neibours(testdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_rs = pd.read_csv('round2/tmp/stack_on_twomodels_no_nan_conflict_v1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_test_rs_neighbours(test_df,test_rs):\n",
    "    new_test_df = test_df.copy()\n",
    "    new_test_df['entity'] = test_rs['key_entity']\n",
    "    neighbours = get_all_neibours(new_test_df)\n",
    "    return neighbours\n",
    "test_rs_neighbours = get_test_rs_neighbours(testdf,test_rs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#统计neigbours标签相同的数量和不同的数量\n",
    "same_neighbours = [(id,e1,e2) for id,e1,e2 in test_rs_neighbours if (test_entity_label_dict[(id,e1)] == test_entity_label_dict[(id,e2)])]\n",
    "diff_neighbours = [(id,e1,e2) for id,e1,e2 in test_rs_neighbours if (test_entity_label_dict[(id,e1)] != test_entity_label_dict[(id,e2)])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9997/9997 [00:41<00:00, 239.16it/s]\n"
     ]
    }
   ],
   "source": [
    "#统计test_rs中至少计算neighbour中一个实体的数量\n",
    "def one_neighbour_choosed(test_rs):\n",
    "    items = []\n",
    "    for id,estr,kestr in tqdm(test_rs[['id','entity','key_entity']].values):\n",
    "        if not isinstance(kestr,str):\n",
    "            continue\n",
    "        for e1 in kestr.split(';'):\n",
    "            for e2 in estr.split(';'):\n",
    "                e1,e2 = exchange(e1,e2)\n",
    "                if (id,e1,e2) in test_neighbours:\n",
    "                    if e1 in kestr and e2 not in kestr:\n",
    "                        items.append((id,e1,e2)) \n",
    "                    if e2 in kestr and e1 not in kestr:\n",
    "                        items.append((id,e1,e2)) \n",
    "    return list(set(items))\n",
    "test_neighours_existed_leat_one = one_neighbour_choosed(test_rs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "231"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_neighours_existed_leat_one )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(15912, '信和财富', '金信网'),\n",
       " (22982, '人人贷', '友信'),\n",
       " (15816, '创鑫', '贷网'),\n",
       " (21394, '平台贷', '金银猫'),\n",
       " (14972, '中植', '新丝路'),\n",
       " (16950, '信和财富', '金信网'),\n",
       " (13339, 'cp', '人人爱家'),\n",
       " (22482, '东银金服', '还我钱'),\n",
       " (16976, '一点车贷', '鸿特信息'),\n",
       " (17225, '团贷网', '惠借款'),\n",
       " (17275, '京东', '卡还'),\n",
       " (18271, '信和大金融', '金信网'),\n",
       " (15570, 'AlipayHK', '港版支付宝'),\n",
       " (14272, '小赢网金', '麻袋财富'),\n",
       " (18510, '信和大金融', '信和财富'),\n",
       " (15677, 'P2P理财', '萌橙理财'),\n",
       " (16068, '东方成安', '长和'),\n",
       " (15594, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (19376, '快还', '钱保姆'),\n",
       " (15260, '信和财富', '金信网'),\n",
       " (17216, '富仁金融', '理财范'),\n",
       " (22194, '快还', '票据宝'),\n",
       " (14002, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (18213, '融金汇银', '银天下'),\n",
       " (21015, '凤凰', '欧凯龙'),\n",
       " (19144, '人人贷', '友信普惠'),\n",
       " (16347, '中信信托有限责任公司', '厦门金海峡投资有限公司'),\n",
       " (20534, '借我', '团贷网'),\n",
       " (15567, '团贷网', '草根'),\n",
       " (18612, '多投', '融宝'),\n",
       " (18674, '时时彩', '炒外汇'),\n",
       " (20302, '荷包金融', '荷包金融（深圳）'),\n",
       " (22228, '360 金融', '小赢科技'),\n",
       " (17320, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (13529, '合众e贷', '网利宝'),\n",
       " (15526, '天猫', '苏宁'),\n",
       " (16794, '360借条', '安心贷'),\n",
       " (16428, '头条', '水松缘'),\n",
       " (15968, '悦花越有', '返利'),\n",
       " (17560, '大丰收', '金融业'),\n",
       " (18888, '宜人贷', '极速借款'),\n",
       " (21279, '头条', '牛牛通'),\n",
       " (20935, '富仁金融', '理财范'),\n",
       " (16570, '团贷网', '惠借款'),\n",
       " (20635, '京东', '商城'),\n",
       " (18878, '时时彩', '炒外汇'),\n",
       " (20882, '商业保理', '快易'),\n",
       " (16678, '荷包金融', '荷包金融（深圳）'),\n",
       " (18943, 'Prodeum', '区块链项目'),\n",
       " (18324, '厦门银行', '团贷网'),\n",
       " (21089, '信和财富', '金信网'),\n",
       " (19639, '人人贷借款', '安居客'),\n",
       " (21001, '安宜投', '微投天下'),\n",
       " (19414, '民蕴财富', '米缸金融'),\n",
       " (17496, '点融', '钱宝网'),\n",
       " (18631, '', '金融圈'),\n",
       " (14777, '荷包金融', '荷包金融（深圳）'),\n",
       " (19414, '余额宝', '民蕴财富'),\n",
       " (20982, '富仁金融', '理财范'),\n",
       " (14156, '京东数科', '蚂蚁金服'),\n",
       " (22228, '360 金融', '简普科技'),\n",
       " (22376, '和合速融', '蚂蚁金服'),\n",
       " (16907, '中植', '新丝路'),\n",
       " (14854, '小赢网金', '麻袋财富'),\n",
       " (20687, '念钱安理财', '靠谱'),\n",
       " (21171, '元泰金服', '靠谱'),\n",
       " (21832, '云付通', '云支付'),\n",
       " (15128, '中航生意贷', '好友邦金服'),\n",
       " (21797, '富仁金融', '理财范'),\n",
       " (15593, '宜信惠民', '宜信普惠'),\n",
       " (15606, '理财范', '量子金融'),\n",
       " (19471, 'p2p产品', '抓钱猫'),\n",
       " (14755, '????达飞云贷', '智慧金融'),\n",
       " (21591, '云联惠', '返利'),\n",
       " (20087, '快速贷', '有财花'),\n",
       " (16493, '快还', '钱保姆'),\n",
       " (18714, 'p2p产品', '抓钱猫'),\n",
       " (18542, '斐讯', '赢基财富'),\n",
       " (16103, '富仁金融', '理财范'),\n",
       " (19406, '快还', '金联储'),\n",
       " (21989, '51人品', '网利宝'),\n",
       " (15474, '中天金融', '汇泉贷'),\n",
       " (15703, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (22520, '中植', '新丝路'),\n",
       " (16959, '翼龙贷', '贷款平台'),\n",
       " (19152, '小花钱包', '捷信'),\n",
       " (15448, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (17743, '乐信', '微贷网'),\n",
       " (18025, '博安杰', '车能贷'),\n",
       " (14129, '', 'UU理财'),\n",
       " (16016, 'SC', '钱多多'),\n",
       " (14533, '掌众财富', '金融工场'),\n",
       " (21517, '东方成安', '长和'),\n",
       " (17108, '团贷网', '惠借款'),\n",
       " (21727, '宜人贷', '极速借款'),\n",
       " (19760, 'adc', '头条'),\n",
       " (20360, '永利宝', '潮流'),\n",
       " (18122, 'EGD网络黄金', '万福币'),\n",
       " (15031, '爱贷网', '返利'),\n",
       " (21483, '东方金钰', '国泰君安'),\n",
       " (14501, '富仁金融', '理财范'),\n",
       " (18204, '富仁金融', '理财范'),\n",
       " (17343, '快还', '钱保姆'),\n",
       " (21428, '平安氧气贷', '拍拍贷'),\n",
       " (22251, '唐小僧', '金融服务'),\n",
       " (16486, '网贷天眼', '钱盆网'),\n",
       " (21664, '壹佰金融', '资金链'),\n",
       " (21989, '投哪网', '挖财'),\n",
       " (22853, '深圳荷包金融信息咨询有限公司', '荷包金融'),\n",
       " (13521, '平太金服', '汇投网'),\n",
       " (13027, '人人贷', '墨迹天气'),\n",
       " (21470, '团贷网', '草根'),\n",
       " (18774, '真融宝', '荷包'),\n",
       " (20151, '杭州友米金融服务外包有限公司', '青蓝金服'),\n",
       " (18434, 'p2p产品', '抓钱猫'),\n",
       " (19157, '德庄金融', '理财网'),\n",
       " (22536, 'e兴金融', '汇泉贷'),\n",
       " (22786, '工商银行', '网络贷'),\n",
       " (19078, '快还', '钱保姆'),\n",
       " (20614, '快还', '钱保姆'),\n",
       " (16288, '快还', '网利宝'),\n",
       " (18490, '泰多米', '融桥咨询'),\n",
       " (22558, '网保', '钱宝'),\n",
       " (15656, '团贷网', '惠借款'),\n",
       " (21999, '快还', '斐讯联璧'),\n",
       " (19576, '云钱袋', '江苏银易贷网络科技有限公司'),\n",
       " (13271, '云付通', '头条'),\n",
       " (18607, '想过', '投之家'),\n",
       " (13926, '信和财富', '金信网'),\n",
       " (15736, '团贷网', '惠借款'),\n",
       " (22838, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (22759, '投融资', '浙江金盾控股集团有限公司'),\n",
       " (15147, 'P2P理财', '蜂融网'),\n",
       " (19023, '快还', '网利宝'),\n",
       " (14850, '伊拉克石油', '无人机'),\n",
       " (22856, '荷包金融', '荷包金融（深圳）'),\n",
       " (18474, ' ', '易投'),\n",
       " (14259, '快还', '钱保姆'),\n",
       " (15710, '信和财富', '金信网'),\n",
       " (17758, '信和财富', '金信网'),\n",
       " (22536, 'e兴金融', '盈盈理财'),\n",
       " (15604, '快还', '票据宝'),\n",
       " (16395, '善心汇', '返利'),\n",
       " (19259, 'fxopen', '外汇经纪商'),\n",
       " (14854, '人人贷', '拍拍贷'),\n",
       " (16969, '中航生意贷', '好友邦金服'),\n",
       " (17246, 'BTC', '比特帮'),\n",
       " (19149, '亚马逊', '当当网'),\n",
       " (19670, '小赢网金', '麻袋财富'),\n",
       " (22335, '国有投资', '昊祥投资'),\n",
       " (15606, '富仁金融', '理财范'),\n",
       " (20200, '快还', '钱保姆'),\n",
       " (22035, '信和财富', '金信网'),\n",
       " (15526, '京东', '淘宝'),\n",
       " (15411, '万得贷', '网通网贷'),\n",
       " (20841, '99金', '伦敦金投资'),\n",
       " (15956, '联璧金融', '钱到'),\n",
       " (17068, '小牛在线', '小牛资本'),\n",
       " (17872, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (22167, '团贷网', '资金链'),\n",
       " (18604, '快还', '钱保姆'),\n",
       " (16678, '深圳荷包金融信息咨询有限公司', '荷包金融'),\n",
       " (13152, '1号店', '聚美优品'),\n",
       " (13780, '人人车', '蚂蚁金服'),\n",
       " (18660, '支付宝', '永利宝'),\n",
       " (16469, '微车融', '资金链'),\n",
       " (20620, '信和财富', '金信网'),\n",
       " (14848, '人人贷', '广信贷'),\n",
       " (19574, '京东数科', '蚂蚁金服'),\n",
       " (17573, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (20133, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司'),\n",
       " (20151, '杭州金碗网络科技有限公司', '青蓝金服'),\n",
       " (22659, 'UU理财', '梦想盈行'),\n",
       " (22927, 'PPmoney', '民蕴财富'),\n",
       " (22740, '上海融金汇银', '银天下'),\n",
       " (15180, '团贷网', '理财范'),\n",
       " (16613, '东方金钰', '中信信托'),\n",
       " (18122, 'EGD网络黄金', 'GCB光彩币'),\n",
       " (20664, '网贷天眼', '美团支付'),\n",
       " (18631, '', 'UU理财'),\n",
       " (22982, '人人贷', '友信普惠'),\n",
       " (20741, '上海融金汇银', '银天下'),\n",
       " (19339, '网信理财', '金融工场'),\n",
       " (16898, '金融投资', '钱包'),\n",
       " (13240, '一鼎金融', '懒人投资'),\n",
       " (16380, '商城', '返利'),\n",
       " (17670, '奇子向钱', '高财生'),\n",
       " (18474, ' ', '易投资'),\n",
       " (16265, '宜人贷', '极速借款'),\n",
       " (16653, '拍拍贷', '马上金融'),\n",
       " (17375, '云联惠', '惠天下'),\n",
       " (17657, '团贷网', '资金链'),\n",
       " (19767, '普资金服', '资金链'),\n",
       " (20935, '理财范', '量子金融'),\n",
       " (22853, '荷包金融', '荷包金融（深圳）'),\n",
       " (13950, '快还', '钱保姆'),\n",
       " (20551, '中望金服', '享金科技'),\n",
       " (17670, '奇子向钱', '高财生金服'),\n",
       " (19421, '德庄金融', '理财网'),\n",
       " (17236, '人人贷', '友信'),\n",
       " (19197, '好易借', '小额贷'),\n",
       " (16347, '中信信托有限责任公司', '百瑞信托有限责任公司'),\n",
       " (18084, '富仁金融', '理财范'),\n",
       " (21989, '51人品', '小赢网金'),\n",
       " (14272, '人人贷', '拍拍贷'),\n",
       " (21266, '理财范', '量子金融'),\n",
       " (14609, '团贷网', '资金链'),\n",
       " (16969, '好友邦金服', '生意贷'),\n",
       " (18612, '多投', '银融宝'),\n",
       " (22139, '团贷网', '惠借款'),\n",
       " (18536, '信和大金融', '金信网'),\n",
       " (21044, '人人贷', '友信'),\n",
       " (15128, '好友邦金服', '生意贷'),\n",
       " (15616, '聚爱财', '钱富通'),\n",
       " (19754, '建设银行', '草根投资'),\n",
       " (15029, '人人愛家', '牛板金'),\n",
       " (19670, '人人贷', '拍拍贷'),\n",
       " (15180, '理财范', '网利宝'),\n",
       " (14129, '', '金融圈'),\n",
       " (13445, '红岭创投', '网贷天眼'),\n",
       " (17216, '理财范', '量子金融'),\n",
       " (18122, '万协云商', '之道出行'),\n",
       " (14208, '99分期', '优分期'),\n",
       " (19930, '快还', '票据宝'),\n",
       " (20995, 'P2P理财', '蜂融网'),\n",
       " (13819, '快还', '钱保姆'),\n",
       " (13651, '微贷网', '银象网'),\n",
       " (21372, '余额宝', '银象网'),\n",
       " (21695, '快还', '钱保姆'),\n",
       " (14055, '德庄金融', '理财网'),\n",
       " (18712, '上海永利宝网络信息科技有限公司', '上海潇谦互联网金融信息服务有限公司')]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#决定给共同出现的entity赋予相同的label\n",
    "test_neighours_existed_leat_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdf['key_entity'] = test_rs['key_entity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "white_entity_list = ['快还','p2p产品','民蕴财富','想过','返利','卡还','小赢网金']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当当网 \n",
      " ????【北京多部门：#618#电商禁先涨价再打折 禁欺骗诱导消费】。为保护消费者合法权益，营造安全放心的消费环境，昨天（14日），北京市市场监管局、北京市价监局、北京市消协等部门对天猫、京东、亚马逊、当当网、国美在线、美团、顺丰优选、今日头条等16家电商企业“6.18”年中促销活动进行集中行政指导和 ????? \n",
      "\n",
      " ????【北京多部门：#618#电商禁先涨价再打折 禁欺骗诱导消费】。为保护消费者合法权益，营造安全放心的消费环境，昨天（14日），北京市市场监管局、北京市价监局、北京市消协等部门对天猫、京东、亚马逊、当当网、国美在线、美团、顺丰优选、今日头条等16家电商企业6.18年中促销活动进行集中行政指导和 ?????\n"
     ]
    }
   ],
   "source": [
    "title,text,key_entity = testdf[testdf['id']==19149][['title','text','key_entity']].values[0]\n",
    "print(key_entity,'\\n',title,'\\n\\n',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "#赋予共同出现的实体同样的分数\n",
    "items = []\n",
    "for id,negative,key_entity in test_rs[['id','negative','key_entity']].values:\n",
    "    if not isinstance(key_entity,str):\n",
    "        items.append((id,negative,key_entity))\n",
    "        continue\n",
    "    ks = key_entity.split(';')\n",
    "    neighbours = [(x[1],x[2]) for x in test_neighours_existed_leat_one if x[0] == id]\n",
    "    for e1,e2 in neighbours:\n",
    "        ks.append(e1)\n",
    "        ks.append(e2)\n",
    "    ks = [e for e in ks if e not in white_entity_list]\n",
    "    items.append((id,negative,';'.join(ks)))\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "same_neighbours = pd.DataFrame(items,columns=['id','negative','key_entity'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "same_neighbours.to_csv('round2/tmp/same_neighbours.csd',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>negative</th>\n",
       "      <th>key_entity</th>\n",
       "      <th>entity</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>469</td>\n",
       "      <td>13470</td>\n",
       "      <td>1</td>\n",
       "      <td>夸客优富;才米公社;点融;和讯网;夸客金融;夸克金融</td>\n",
       "      <td>和讯网;才米公社;点融;夸克金融;夸客金融;夸客优富</td>\n",
       "      <td>夸克金融法人投案自首,旗下才米公社待收超38亿!</td>\n",
       "      <td>{IMG:1} 又一起百亿平台爆雷，3.6万名投资人受波及，公司法人主动投案自首，平台非吸...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>555</td>\n",
       "      <td>13556</td>\n",
       "      <td>1</td>\n",
       "      <td>中瑞财富;米金社</td>\n",
       "      <td>中瑞财富;米金社</td>\n",
       "      <td>????#亚洲财经# 【杭州P2P米金社和北京P2P中瑞财富同日宣布退出】 7月10日，杭州...</td>\n",
       "      <td>????#亚洲财经# 【杭州P2P米金社和北京P2P中瑞财富同日宣布退出】 7月10日，杭州...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1564</td>\n",
       "      <td>14565</td>\n",
       "      <td>1</td>\n",
       "      <td>夸客优富;才米公社;点融;和讯网;夸客金融;夸克金融</td>\n",
       "      <td>和讯网;才米公社;点融;夸克金融;夸客金融;夸客优富</td>\n",
       "      <td>夸克金融法人投案自首,才米公社会怎样?</td>\n",
       "      <td>{IMG:1} #侦探导语#绝大部分投资人，其实根本 不了解平台的资产情况，当然，平台也不...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1878</td>\n",
       "      <td>14879</td>\n",
       "      <td>1</td>\n",
       "      <td>米金社;中瑞财富</td>\n",
       "      <td>中瑞财富;米金社</td>\n",
       "      <td>与华为合作：摩洛哥成为首个实现5G全覆盖的国家；又一家P2P倒台，米金社宣布清盘；中瑞财富宣...</td>\n",
       "      <td>点击上方 狮城集华圈，选择置顶公众号 小秘微信：c-circle | Whatsapp: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260</td>\n",
       "      <td>15261</td>\n",
       "      <td>1</td>\n",
       "      <td>安心快线;昌兰财富;拉拉财富;瑞徳霖投资;曾舜财富;华瀚曌天;中科国睿投资;中国建银;华福嘉...</td>\n",
       "      <td>国祥金融;中金万通;安心快线;正和投资;曾舜财富;中诚兴融;中科国睿投资;正泰投资;掌为投资...</td>\n",
       "      <td>【10月08日预警】网曝诚信贷部分项目逾期！诚信贷、合本投资、创瑞投资、中金万通、盛新财富</td>\n",
       "      <td>【10月08日预警】网曝诚信贷部分项目逾期！诚信贷、合本投资、创瑞投资、中金万通、盛新财富...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4344</td>\n",
       "      <td>17345</td>\n",
       "      <td>1</td>\n",
       "      <td>米金社;小九金服</td>\n",
       "      <td>米金社;小九金服</td>\n",
       "      <td>假标嫌疑的高返平台米金社CEO韦鹏良曾是小九金服高管</td>\n",
       "      <td>{IMG:1} 网贷的信任危机已经正式从上海扩散到了杭州，小九、牛板金陆续暴雷，P2P这个...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8527</td>\n",
       "      <td>21530</td>\n",
       "      <td>1</td>\n",
       "      <td>杭州和讯创富股权投资基金合伙企业;夸客优富;才米公社;点融网;上海夸客金融信息服务有限公司;...</td>\n",
       "      <td>暖薪贷;夸时贷;联璧金融;才米公社;杭州和讯创富股权投资基金合伙企业;点融网;上海联璧电子科...</td>\n",
       "      <td>【曝3条】夸克金融法人投案自首,才米公社会怎样?</td>\n",
       "      <td>{IMG:1} 一、夸克金融法人投案自首，才米公社会怎样？ 绝大部分投资人，其实根本 不了...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8983</td>\n",
       "      <td>21986</td>\n",
       "      <td>1</td>\n",
       "      <td>车贷宝;米金社;小九金服</td>\n",
       "      <td>米金社;车贷宝;小九金服</td>\n",
       "      <td>CC网贷社区：假标嫌疑的高返平台米金社CEO曾是小九金服高管</td>\n",
       "      <td>米金社的股东结构异常简单，大股东朱建新控股80%，朱建新此人在商业世界默默无闻，并无其他任...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  negative                                         key_entity  \\\n",
       "469   13470         1                         夸客优富;才米公社;点融;和讯网;夸客金融;夸克金融   \n",
       "555   13556         1                                           中瑞财富;米金社   \n",
       "1564  14565         1                         夸客优富;才米公社;点融;和讯网;夸客金融;夸克金融   \n",
       "1878  14879         1                                           米金社;中瑞财富   \n",
       "2260  15261         1  安心快线;昌兰财富;拉拉财富;瑞徳霖投资;曾舜财富;华瀚曌天;中科国睿投资;中国建银;华福嘉...   \n",
       "4344  17345         1                                           米金社;小九金服   \n",
       "8527  21530         1  杭州和讯创富股权投资基金合伙企业;夸客优富;才米公社;点融网;上海夸客金融信息服务有限公司;...   \n",
       "8983  21986         1                                       车贷宝;米金社;小九金服   \n",
       "\n",
       "                                                 entity  \\\n",
       "469                          和讯网;才米公社;点融;夸克金融;夸客金融;夸客优富   \n",
       "555                                            中瑞财富;米金社   \n",
       "1564                         和讯网;才米公社;点融;夸克金融;夸客金融;夸客优富   \n",
       "1878                                           中瑞财富;米金社   \n",
       "2260  国祥金融;中金万通;安心快线;正和投资;曾舜财富;中诚兴融;中科国睿投资;正泰投资;掌为投资...   \n",
       "4344                                           米金社;小九金服   \n",
       "8527  暖薪贷;夸时贷;联璧金融;才米公社;杭州和讯创富股权投资基金合伙企业;点融网;上海联璧电子科...   \n",
       "8983                                       米金社;车贷宝;小九金服   \n",
       "\n",
       "                                                  title  \\\n",
       "469                            夸克金融法人投案自首,旗下才米公社待收超38亿!   \n",
       "555   ????#亚洲财经# 【杭州P2P米金社和北京P2P中瑞财富同日宣布退出】 7月10日，杭州...   \n",
       "1564                                夸克金融法人投案自首,才米公社会怎样?   \n",
       "1878  与华为合作：摩洛哥成为首个实现5G全覆盖的国家；又一家P2P倒台，米金社宣布清盘；中瑞财富宣...   \n",
       "2260      【10月08日预警】网曝诚信贷部分项目逾期！诚信贷、合本投资、创瑞投资、中金万通、盛新财富   \n",
       "4344                         假标嫌疑的高返平台米金社CEO韦鹏良曾是小九金服高管   \n",
       "8527                           【曝3条】夸克金融法人投案自首,才米公社会怎样?   \n",
       "8983                     CC网贷社区：假标嫌疑的高返平台米金社CEO曾是小九金服高管   \n",
       "\n",
       "                                                   text  \n",
       "469    {IMG:1} 又一起百亿平台爆雷，3.6万名投资人受波及，公司法人主动投案自首，平台非吸...  \n",
       "555   ????#亚洲财经# 【杭州P2P米金社和北京P2P中瑞财富同日宣布退出】 7月10日，杭州...  \n",
       "1564   {IMG:1} #侦探导语#绝大部分投资人，其实根本 不了解平台的资产情况，当然，平台也不...  \n",
       "1878   点击上方 狮城集华圈，选择置顶公众号 小秘微信：c-circle | Whatsapp: ...  \n",
       "2260   【10月08日预警】网曝诚信贷部分项目逾期！诚信贷、合本投资、创瑞投资、中金万通、盛新财富...  \n",
       "4344   {IMG:1} 网贷的信任危机已经正式从上海扩散到了杭州，小九、牛板金陆续暴雷，P2P这个...  \n",
       "8527   {IMG:1} 一、夸克金融法人投案自首，才米公社会怎样？ 绝大部分投资人，其实根本 不了...  \n",
       "8983   米金社的股东结构异常简单，大股东朱建新控股80%，朱建新此人在商业世界默默无闻，并无其他任...  "
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_rs[test_rs['key_entity'].map(lambda x:('社' in str(x)) )]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "es = set()\n",
    "for estr in train['entity'].values:\n",
    "    for e in str(estr).split(';'):\n",
    "        es.add(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "kes = set()\n",
    "for estr in train['key_entity'].values:\n",
    "    for e in str(estr).split(';'):\n",
    "        kes.add(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>negative</th>\n",
       "      <th>key_entity</th>\n",
       "      <th>entity</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>13009</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>深圳民富集团有限公司;利民网</td>\n",
       "      <td>利民网互金预警网评测，政策风险大于一切！</td>\n",
       "      <td>利民网互金预警网评测，，政策风险大于一切！平台名称：利民网投资参考： 基本情况：利民网 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>13011</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>现金贷;小贷;猎豹移动;小额贷;速贷;猎豹极速贷;武汉市金山互联网小额贷款有限责任公司;小额...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>????公开信息显示，2017年开展现金贷业务的猎豹极速贷并未拥有小贷牌照，不过，猎豹移动旗...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>13014</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>收钱吧;钱吧</td>\n",
       "      <td>NaN</td>\n",
       "      <td>????作为国内最早深耕线下移动支付服务的服务商之一，收钱吧最早推出了聚合收款的创新模式，并...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>13017</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>诸葛理财</td>\n",
       "      <td>诸葛理财P2P网贷平台2018/06/04数据分析</td>\n",
       "      <td>诸葛理财P2P网贷平台2018/06/04数据分析 2018-06-05???来源：第一网...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>13028</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>宜人贷</td>\n",
       "      <td>中国概念股周二收盘多数上涨 宜人贷飙涨逾17%</td>\n",
       "      <td>美股周二收高，总统大选辩论结果与一些好于预期的经济报告令市场得到支撑。 　　美东时间9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>13033</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>东方城;一带一路;拆分模式</td>\n",
       "      <td>NaN</td>\n",
       "      <td>马来西亚与中国一带一路项目.东方城.依托实体.通过拆分模式建立的大数据.??</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>13035</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>返利;云联;云联惠</td>\n",
       "      <td>NaN</td>\n",
       "      <td>针对云联惠在去库存方面获得的众多关注，何智斌博士主讲《云联惠消费返还是房就云联惠与房W去库存...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>13037</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>米房无忧借条;无忧借条;借贷宝</td>\n",
       "      <td>魅力孕妇这世界上不缺漂亮今借到姑娘，不缺有钱有凭证富人，更不缺早已泛滥的借贷宝快餐式爱情，真...</td>\n",
       "      <td>魅力孕妇这世界上不缺漂亮今借到姑娘，不缺有钱有凭证富人，更不缺早已泛滥的借贷宝快餐式爱情，真...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>13038</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>宜人理财;宜人贷;宜信</td>\n",
       "      <td>宜人贷可靠吗 宜人贷借一万一年还多少钱？</td>\n",
       "      <td>网购平台那么多，有时候真的不知道该选择哪个比较好？大家觉得宜人贷怎么样呢？确实，有不少人希...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>13040</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>小额贷;惠花花</td>\n",
       "      <td>NaN</td>\n",
       "      <td>（彩蛋：惠花花是主要满足用户在日常生活中的小额贷款、借款、借贷需，解决大家短时间内需要钱的紧张时期</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>13047</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>快付;医快付</td>\n",
       "      <td>NaN</td>\n",
       "      <td>县人民医院、消化内科、下午……提前一天，许正卿就打开医快付APP(一款融合医保的移动就医AP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>13051</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>随心贷;中原银行</td>\n",
       "      <td>中原银行首笔“随心贷”业务落户南阳分行</td>\n",
       "      <td>8月11日，中原银行随心贷产品成功上线，中原银行南阳分行经过不懈努力，实现业务成功审...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>13056</td>\n",
       "      <td>1</td>\n",
       "      <td>佰亿猫;博安杰;翡翠岛;人人爱家;啄米理财;投融家;萌小薪;壹盐双创;五星财富;合时代;投之...</td>\n",
       "      <td>蓝天格锐;唐小僧;萝卜智投(启蓝);佰亿猫;利民网;钱内助;金坳财富;惠盈理财;长富理财;金...</td>\n",
       "      <td>公安部开通56家P2P非法集资登记通道,包含蓝天格瑞、唐小僧、投之家、多多理财、银票网、人人...</td>\n",
       "      <td>{IMG:1} 网贷之家观察到，就在9月10日，公安部在其官网非法集资案件投资人信息登记平...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>13059</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>首金网</td>\n",
       "      <td>首金网P2P网贷平台2018年06月01日数据分析</td>\n",
       "      <td>首金网P2P网贷平台2018年06月01日数据分析 2018-06-02???来源：第一网...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>13061</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>蚂蚁理财;稳赚</td>\n",
       "      <td>????#天津爆料# @平安天津：日前，天津警方接到一女子王某报警称：2019年5月10日，...</td>\n",
       "      <td>????#天津爆料# @平安天津：日前，天津警方接到一女子王某报警称：2019年5月10日，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>13070</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>米房无忧借条;借贷宝</td>\n",
       "      <td>她和他是男女朋友，她爱他，但是两人在一起时她就从来没有开心过。分手后的一次偶然机会，她穿越回...</td>\n",
       "      <td>她和他是男女朋友，她爱他，但是两人在一起时她就从来没有开心过。分手后的一次偶然机会，她穿越回...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>13073</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>陆金所;消费金融</td>\n",
       "      <td>NaN</td>\n",
       "      <td>????陆金所首次实现全年整体盈利????平安年报显示，2017年，在财富管理、消费金融、机...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>13075</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>积木盒子;宜人贷;汇银富通;铜板街</td>\n",
       "      <td>还有“睡后收入“这种操作？汇银富通、积木盒子、铜板街、宜人贷了解一下的相关新闻汇总</td>\n",
       "      <td>还有睡后收入这种操作？汇银富通、积木盒子、铜板街、宜人贷了解一下的相关新闻汇总</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>13076</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>消费金融;神州优车;河北幸福消费金融公司;幸福消费金融</td>\n",
       "      <td>NaN</td>\n",
       "      <td>????财报还披露，2017年神州优车先后认购优车产业基金，参与发起设立河北幸福消费金融公司...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83</td>\n",
       "      <td>13084</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>牛板金;投之家;草根投资</td>\n",
       "      <td>????//@用户7058182311://@用户vfcx8zwltt:@用户vfcx8zw...</td>\n",
       "      <td>????//@用户7058182311://@用户vfcx8zwltt:@用户vfcx8zw...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  negative                                         key_entity  \\\n",
       "8   13009         0                                                NaN   \n",
       "10  13011         0                                                NaN   \n",
       "13  13014         0                                                NaN   \n",
       "16  13017         0                                                NaN   \n",
       "27  13028         0                                                NaN   \n",
       "32  13033         0                                                NaN   \n",
       "34  13035         0                                                NaN   \n",
       "36  13037         0                                                NaN   \n",
       "37  13038         0                                                NaN   \n",
       "39  13040         0                                                NaN   \n",
       "46  13047         0                                                NaN   \n",
       "50  13051         0                                                NaN   \n",
       "55  13056         1  佰亿猫;博安杰;翡翠岛;人人爱家;啄米理财;投融家;萌小薪;壹盐双创;五星财富;合时代;投之...   \n",
       "58  13059         0                                                NaN   \n",
       "60  13061         0                                                NaN   \n",
       "69  13070         0                                                NaN   \n",
       "72  13073         0                                                NaN   \n",
       "74  13075         0                                                NaN   \n",
       "75  13076         0                                                NaN   \n",
       "83  13084         0                                                NaN   \n",
       "\n",
       "                                               entity  \\\n",
       "8                                      深圳民富集团有限公司;利民网   \n",
       "10  现金贷;小贷;猎豹移动;小额贷;速贷;猎豹极速贷;武汉市金山互联网小额贷款有限责任公司;小额...   \n",
       "13                                             收钱吧;钱吧   \n",
       "16                                               诸葛理财   \n",
       "27                                                宜人贷   \n",
       "32                                      东方城;一带一路;拆分模式   \n",
       "34                                          返利;云联;云联惠   \n",
       "36                                    米房无忧借条;无忧借条;借贷宝   \n",
       "37                                        宜人理财;宜人贷;宜信   \n",
       "39                                            小额贷;惠花花   \n",
       "46                                             快付;医快付   \n",
       "50                                           随心贷;中原银行   \n",
       "55  蓝天格锐;唐小僧;萝卜智投(启蓝);佰亿猫;利民网;钱内助;金坳财富;惠盈理财;长富理财;金...   \n",
       "58                                                首金网   \n",
       "60                                            蚂蚁理财;稳赚   \n",
       "69                                         米房无忧借条;借贷宝   \n",
       "72                                           陆金所;消费金融   \n",
       "74                                  积木盒子;宜人贷;汇银富通;铜板街   \n",
       "75                        消费金融;神州优车;河北幸福消费金融公司;幸福消费金融   \n",
       "83                                       牛板金;投之家;草根投资   \n",
       "\n",
       "                                                title  \\\n",
       "8                                利民网互金预警网评测，政策风险大于一切！   \n",
       "10                                                NaN   \n",
       "13                                                NaN   \n",
       "16                          诸葛理财P2P网贷平台2018/06/04数据分析   \n",
       "27                            中国概念股周二收盘多数上涨 宜人贷飙涨逾17%   \n",
       "32                                                NaN   \n",
       "34                                                NaN   \n",
       "36  魅力孕妇这世界上不缺漂亮今借到姑娘，不缺有钱有凭证富人，更不缺早已泛滥的借贷宝快餐式爱情，真...   \n",
       "37                               宜人贷可靠吗 宜人贷借一万一年还多少钱？   \n",
       "39                                                NaN   \n",
       "46                                                NaN   \n",
       "50                                中原银行首笔“随心贷”业务落户南阳分行   \n",
       "55  公安部开通56家P2P非法集资登记通道,包含蓝天格瑞、唐小僧、投之家、多多理财、银票网、人人...   \n",
       "58                          首金网P2P网贷平台2018年06月01日数据分析   \n",
       "60  ????#天津爆料# @平安天津：日前，天津警方接到一女子王某报警称：2019年5月10日，...   \n",
       "69  她和他是男女朋友，她爱他，但是两人在一起时她就从来没有开心过。分手后的一次偶然机会，她穿越回...   \n",
       "72                                                NaN   \n",
       "74          还有“睡后收入“这种操作？汇银富通、积木盒子、铜板街、宜人贷了解一下的相关新闻汇总   \n",
       "75                                                NaN   \n",
       "83  ????//@用户7058182311://@用户vfcx8zwltt:@用户vfcx8zw...   \n",
       "\n",
       "                                                 text  \n",
       "8   　 利民网互金预警网评测，，政策风险大于一切！平台名称：利民网投资参考： 基本情况：利民网 ...  \n",
       "10  ????公开信息显示，2017年开展现金贷业务的猎豹极速贷并未拥有小贷牌照，不过，猎豹移动旗...  \n",
       "13  ????作为国内最早深耕线下移动支付服务的服务商之一，收钱吧最早推出了聚合收款的创新模式，并...  \n",
       "16   诸葛理财P2P网贷平台2018/06/04数据分析 2018-06-05???来源：第一网...  \n",
       "27   　　美股周二收高，总统大选辩论结果与一些好于预期的经济报告令市场得到支撑。 　　美东时间9...  \n",
       "32             马来西亚与中国一带一路项目.东方城.依托实体.通过拆分模式建立的大数据.??  \n",
       "34  针对云联惠在去库存方面获得的众多关注，何智斌博士主讲《云联惠消费返还是房就云联惠与房W去库存...  \n",
       "36  魅力孕妇这世界上不缺漂亮今借到姑娘，不缺有钱有凭证富人，更不缺早已泛滥的借贷宝快餐式爱情，真...  \n",
       "37   网购平台那么多，有时候真的不知道该选择哪个比较好？大家觉得宜人贷怎么样呢？确实，有不少人希...  \n",
       "39  （彩蛋：惠花花是主要满足用户在日常生活中的小额贷款、借款、借贷需，解决大家短时间内需要钱的紧张时期  \n",
       "46  县人民医院、消化内科、下午……提前一天，许正卿就打开医快付APP(一款融合医保的移动就医AP...  \n",
       "50  　　　 8月11日，中原银行随心贷产品成功上线，中原银行南阳分行经过不懈努力，实现业务成功审...  \n",
       "55   {IMG:1} 网贷之家观察到，就在9月10日，公安部在其官网非法集资案件投资人信息登记平...  \n",
       "58   首金网P2P网贷平台2018年06月01日数据分析 2018-06-02???来源：第一网...  \n",
       "60  ????#天津爆料# @平安天津：日前，天津警方接到一女子王某报警称：2019年5月10日，...  \n",
       "69  她和他是男女朋友，她爱他，但是两人在一起时她就从来没有开心过。分手后的一次偶然机会，她穿越回...  \n",
       "72  ????陆金所首次实现全年整体盈利????平安年报显示，2017年，在财富管理、消费金融、机...  \n",
       "74            还有睡后收入这种操作？汇银富通、积木盒子、铜板街、宜人贷了解一下的相关新闻汇总  \n",
       "75  ????财报还披露，2017年神州优车先后认购优车产业基金，参与发起设立河北幸福消费金融公司...  \n",
       "83  ????//@用户7058182311://@用户vfcx8zwltt:@用户vfcx8zw...  "
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_max_index(estr,text,kestr):\n",
    "    if not isinstance(kestr,str):\n",
    "        return True\n",
    "    poss = []\n",
    "    text = str(text)\n",
    "    for e in str(estr).split(';'):\n",
    "        pos = text.find(e)\n",
    "        if pos > 400 and e not in kestr:\n",
    "            return True\n",
    "        \n",
    "    return False\n",
    "test_rs[test_rs[['entity','text','key_entity']].apply(lambda x: get_max_index(x[0],x[1],x[2]),axis=1)].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "#规则2：对于用于推荐的文本，模型分类不准确\n",
    "#查看训练集中包含“推荐”的样本\n",
    "def has_recommand_wors(title,text,words):\n",
    "    text =str(title) + str(text)\n",
    "    for word in words:\n",
    "        if word in text:\n",
    "            return True\n",
    "    return False\n",
    "recommand_df = train[train[['title','text']].apply(lambda x:has_recommand_wors(x[0],x[1],['推荐']),axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 148, 0: 92})"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(recommand_df['negative'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['原标题：证券时报e公司讯，老百姓(603883)7月23日晚间公告，持股29.78%的股东泽星投资计划减持不超过8533.87万股，即不超过公司股份总数的29.78%。 最新网 www.zuinow.com 2019-07-29 05:12:59 热点图文美国橄榄球球员汤姆·布雷迪带着6岁小女儿在悬崖边跳水，网友吵翻高血压饮食、预防、危害及血压测量的4大选择 推荐新闻中民投已完成18民生投资SCP007超短融兑付2019中国最受欢迎景区TOP100揭晓，黄山位居榜首 本类推荐中民投已完成18民生投资SCP007超短融兑付2019中国最受欢迎景区TOP100揭晓，黄山位居榜首中国电信去年业绩增长拟斥巨资投入5G建设'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommand_df[recommand_df['id']=='48402b7d']['text'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>entity</th>\n",
       "      <th>negative</th>\n",
       "      <th>key_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>0f37c730</td>\n",
       "      <td>利海资源定增“罗生门”：赛伯乐投资旗下公司陷入兑付危机</td>\n",
       "      <td>{IMG:1} {IMG:2} 利海资源定增罗生门：赛伯乐投资旗下公司陷入兑付危机 来源:...</td>\n",
       "      <td>赛伯乐绿科;德威公司;新余铭沃;乐投资;赛伯乐投资</td>\n",
       "      <td>1</td>\n",
       "      <td>赛伯乐绿科;德威公司;新余铭沃;赛伯乐投资</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>108</td>\n",
       "      <td>48402b7d</td>\n",
       "      <td>老百姓：泽星投资拟减持不超29.78%股份</td>\n",
       "      <td>原标题：证券时报e公司讯，老百姓(603883)7月23日晚间公告，持股29.78%的股东泽...</td>\n",
       "      <td>中国电信;超短融兑付;泽星投资</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>167</td>\n",
       "      <td>0cb3135e</td>\n",
       "      <td>蜂融网P2P理财：想要靠理财增收就要具备风险防范的能力！</td>\n",
       "      <td>随着人们收入的增长，生活条件也日益见好，人往高处走，有了闲钱自然就会想到拥有更多的闲钱...</td>\n",
       "      <td>P2P理财;蜂融网</td>\n",
       "      <td>1</td>\n",
       "      <td>蜂融网</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>175</td>\n",
       "      <td>5ae9b4d8</td>\n",
       "      <td>????网利宝诈骗#由杜海涛代言并且湖南卫视和爱奇艺平台倾力推荐的明星平台网利宝投资理财，目...</td>\n",
       "      <td>????网利宝诈骗#由杜海涛代言并且湖南卫视和爱奇艺平台倾力推荐的明星平台网利宝投资理财，目...</td>\n",
       "      <td>资金链;网利宝;????网利宝</td>\n",
       "      <td>1</td>\n",
       "      <td>????网利宝</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>194</td>\n",
       "      <td>e6f186be</td>\n",
       "      <td>安心保险关于米缸金融逾期公告函、怎么看！</td>\n",
       "      <td>{IMG:1}点击箭头处蓝色字，关注我们哦！！来源：微信公众号/网贷内幕据投资人反映，米缸网...</td>\n",
       "      <td>米缸金融;安心保险</td>\n",
       "      <td>1</td>\n",
       "      <td>米缸金融</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12816</td>\n",
       "      <td>12818</td>\n",
       "      <td>NaN</td>\n",
       "      <td>诺亚财富推荐这款理财计划的时候，没有任何人告诉我万科昆明地块是个没边的事，明显是一种欺诈</td>\n",
       "      <td>诺亚;理财计划</td>\n",
       "      <td>1</td>\n",
       "      <td>诺亚</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12900</td>\n",
       "      <td>12902</td>\n",
       "      <td>NaN</td>\n",
       "      <td>或可回归项目类信托计划模式评论（0）记者武晓蒙2018年04月25日18:50像做项目类信托...</td>\n",
       "      <td>善林金融;财新</td>\n",
       "      <td>1</td>\n",
       "      <td>善林金融</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12912</td>\n",
       "      <td>12914</td>\n",
       "      <td>《奇妙的食光》发声明道歉,安全透明理财靠白金钱包、秉利金服、乙松金服、91飞猫、懿金服</td>\n",
       "      <td>随着经济的发展，互联网金融行业的规模也在不断扩大。除了传统理财方式之外，P2P网贷也逐渐被...</td>\n",
       "      <td>乙松金服;陆金所;小赢理财;91飞猫;懿金服;白金钱包;星星贷;秉利金服</td>\n",
       "      <td>1</td>\n",
       "      <td>乙松金服;陆金所;小赢理财;91飞猫;白金钱包;懿金服;星星贷;秉利金服</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12944</td>\n",
       "      <td>12946</td>\n",
       "      <td>问专家（第13期）：银湖网老板进军新能源？玖融清盘！玖富哪些标有履约险？          ...</td>\n",
       "      <td>「专家问答」是互金跟投开设的问答栏目，跟投专家每天都将回复3-5条来自投友们的问题。大家有什...</td>\n",
       "      <td>投哪网;易通贷;多多理财;图腾贷;人人聚财;团贷网;银湖网;宜信;果树财富;微贷网;沪商财富...</td>\n",
       "      <td>1</td>\n",
       "      <td>沪商财富;图腾贷;玖融网;蜜蜂聚财;泓申金服;果树财富</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12982</td>\n",
       "      <td>12984</td>\n",
       "      <td>越秀金控(000987)非公开发行股份收购广州证券32.765%股份 广州证券将成全资子公司</td>\n",
       "      <td>越秀金控(000987)非公开发行股份收购广州证券32.765%股份 广州证券将成全资子公...</td>\n",
       "      <td>广州证券;越秀金控</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                              title  \\\n",
       "72     0f37c730                        利海资源定增“罗生门”：赛伯乐投资旗下公司陷入兑付危机   \n",
       "108    48402b7d                              老百姓：泽星投资拟减持不超29.78%股份   \n",
       "167    0cb3135e                       蜂融网P2P理财：想要靠理财增收就要具备风险防范的能力！   \n",
       "175    5ae9b4d8  ????网利宝诈骗#由杜海涛代言并且湖南卫视和爱奇艺平台倾力推荐的明星平台网利宝投资理财，目...   \n",
       "194    e6f186be                               安心保险关于米缸金融逾期公告函、怎么看！   \n",
       "...         ...                                                ...   \n",
       "12816     12818                                                NaN   \n",
       "12900     12902                                                NaN   \n",
       "12912     12914        《奇妙的食光》发声明道歉,安全透明理财靠白金钱包、秉利金服、乙松金服、91飞猫、懿金服   \n",
       "12944     12946  问专家（第13期）：银湖网老板进军新能源？玖融清盘！玖富哪些标有履约险？          ...   \n",
       "12982     12984     越秀金控(000987)非公开发行股份收购广州证券32.765%股份 广州证券将成全资子公司   \n",
       "\n",
       "                                                    text  \\\n",
       "72      {IMG:1} {IMG:2} 利海资源定增罗生门：赛伯乐投资旗下公司陷入兑付危机 来源:...   \n",
       "108    原标题：证券时报e公司讯，老百姓(603883)7月23日晚间公告，持股29.78%的股东泽...   \n",
       "167     　　随着人们收入的增长，生活条件也日益见好，人往高处走，有了闲钱自然就会想到拥有更多的闲钱...   \n",
       "175    ????网利宝诈骗#由杜海涛代言并且湖南卫视和爱奇艺平台倾力推荐的明星平台网利宝投资理财，目...   \n",
       "194    {IMG:1}点击箭头处蓝色字，关注我们哦！！来源：微信公众号/网贷内幕据投资人反映，米缸网...   \n",
       "...                                                  ...   \n",
       "12816     　　诺亚财富推荐这款理财计划的时候，没有任何人告诉我万科昆明地块是个没边的事，明显是一种欺诈   \n",
       "12900  或可回归项目类信托计划模式评论（0）记者武晓蒙2018年04月25日18:50像做项目类信托...   \n",
       "12912   随着经济的发展，互联网金融行业的规模也在不断扩大。除了传统理财方式之外，P2P网贷也逐渐被...   \n",
       "12944  「专家问答」是互金跟投开设的问答栏目，跟投专家每天都将回复3-5条来自投友们的问题。大家有什...   \n",
       "12982   越秀金控(000987)非公开发行股份收购广州证券32.765%股份 广州证券将成全资子公...   \n",
       "\n",
       "                                                  entity  negative  \\\n",
       "72                             赛伯乐绿科;德威公司;新余铭沃;乐投资;赛伯乐投资         1   \n",
       "108                                      中国电信;超短融兑付;泽星投资         0   \n",
       "167                                            P2P理财;蜂融网         1   \n",
       "175                                      资金链;网利宝;????网利宝         1   \n",
       "194                                            米缸金融;安心保险         1   \n",
       "...                                                  ...       ...   \n",
       "12816                                            诺亚;理财计划         1   \n",
       "12900                                            善林金融;财新         1   \n",
       "12912               乙松金服;陆金所;小赢理财;91飞猫;懿金服;白金钱包;星星贷;秉利金服         1   \n",
       "12944  投哪网;易通贷;多多理财;图腾贷;人人聚财;团贷网;银湖网;宜信;果树财富;微贷网;沪商财富...         1   \n",
       "12982                                          广州证券;越秀金控         0   \n",
       "\n",
       "                                 key_entity  \n",
       "72                    赛伯乐绿科;德威公司;新余铭沃;赛伯乐投资  \n",
       "108                                     NaN  \n",
       "167                                     蜂融网  \n",
       "175                                 ????网利宝  \n",
       "194                                    米缸金融  \n",
       "...                                     ...  \n",
       "12816                                    诺亚  \n",
       "12900                                  善林金融  \n",
       "12912  乙松金服;陆金所;小赢理财;91飞猫;白金钱包;懿金服;星星贷;秉利金服  \n",
       "12944           沪商财富;图腾贷;玖融网;蜜蜂聚财;泓申金服;果树财富  \n",
       "12982                                   NaN  \n",
       "\n",
       "[240 rows x 6 columns]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommand_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_rs['title'] = testdf['title']\n",
    "test_rs['text'] = testdf['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_recommand_df = test_rs[test_rs[['title','text']].apply(lambda x:has_recommand_wors(x[0],x[1],['推荐']),axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 214, 0: 78})"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(test_recommand_df['negative'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['遇见', '你', '真的', '是', '太好了']\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "luca = jieba.cut('遇见你真的是太好了')\n",
    "print(list(luca))#直接使用jieba.cut()返回的不是列表，需要用list()或set(),''.join()等方式输出，或使用jieba.lcut()\n",
    "text = '遇见你真的是太好了'\n",
    "import jieba.analyse\n",
    "# for x, w in jieba.analyse.extract_tags(text):#可以再添加一个参数指定输出个数\n",
    "#     print(x, w)#直接输出关键词和词频"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rm_char1(text1):\n",
    "    text1 = re.sub('\\u3000', '', text1)    \n",
    "    return text1\n",
    "    \n",
    "def rm_char2(text2):\n",
    "    text2 = re.sub('\\xa0', '', text2)\n",
    "    return text2\n",
    "\n",
    "def get_stop_words():\n",
    "    # stop_words中，每行放一个停用词，以\\n分隔\n",
    "    with open('/home/fyyc/下载/stop.txt','r', encoding = 'utf8') as f:\n",
    "        file = f.read().split('\\n')\n",
    "    return set(file)\n",
    "\n",
    "def rm_tokens(words): # 去掉一些停用词和数字\n",
    "    words_list = list(words)\n",
    "    stop_words = get_stop_words()\n",
    "    for i in range(words_list.__len__())[::-1]:\n",
    "        if words_list[i] in stop_words: # 去除停用词\n",
    "            words_list.pop(i)\n",
    "        elif words_list[i].isdigit():\n",
    "            words_list.pop(i)\n",
    "    return words_list\n",
    "\n",
    "def convert_text_to_wordlist(str_doc):\n",
    "    # 分词的主要方法\n",
    "    sent_list = str_doc.split('\\n')\n",
    "    sent_list = map(rm_char1, sent_list) # 去掉一些字符，例如\\u3000\n",
    "    sent_list = map(rm_char2, sent_list) # 去掉\\xa0\n",
    "    word_2dlist = [rm_tokens(jieba.cut(part)) for part in sent_list] # 分词\n",
    "    word_list = sum(word_2dlist,[])\n",
    "    return word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "import jieba.analyse\n",
    "import csv\n",
    "from sklearn import feature_extraction  \n",
    "\n",
    "def get_dataset():\n",
    "    data, targetdata = [], []\n",
    "    for title,text,negative in recommand_df[['title','text','negative']].values:\n",
    "        text = str(title)+str(text)\n",
    "        words = ' '.join(jieba.lcut(text))\n",
    "        data.append(words)\n",
    "        targetdata.append(negative)\n",
    "    return data,targetdata\n",
    "def get_test_dataset():\n",
    "    data = []\n",
    "    for title,text in test_recommand_df[['title','text']].values:\n",
    "        text = str(title)+str(text)\n",
    "        words = ' '.join(jieba.lcut(text))\n",
    "        data.append(words)\n",
    "    return data\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn import svm\n",
    "\n",
    "def data_pro():\n",
    "    data_,target_train = get_dataset()\n",
    "    test_data = get_test_dataset()\n",
    "    \n",
    "    v = TfidfVectorizer()\n",
    "    train_data = v.fit_transform(data_)\n",
    "    test_data = v.transform(test_data)\n",
    "    \n",
    "    return train_data,target_train, test_data\n",
    "\n",
    "train_data,target_train, test_data= data_pro()   \n",
    "clf = MultinomialNB(alpha=0.01)  \n",
    "clf.fit(train_data,target_train)   \n",
    "pred = clf.predict(test_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
